package com.ai.ly.goodsautocat.autocat;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.ai.ly.goodsautocat.common.Constants;
import com.ai.ly.goodsautocat.common.SampleFactory;
import com.ai.ly.goodsautocat.common.SampleInfoBuilderFactory;
import com.ai.ly.goodsautocat.util.FileUtil;
import com.ai.ly.goodsautocat.vo.CatInfo;
import com.ai.ly.goodsautocat.vo.SampleMapHolder;
import com.ai.ly.goodsautocat.vo.TrainningSampleVO;
import com.ai.ly.goodsautocat.vo.VaildResultVO;

import de.bwaldvogel.liblinear.Feature;
import de.bwaldvogel.liblinear.FeatureNode;
import de.bwaldvogel.liblinear.Linear;
import de.bwaldvogel.liblinear.Model;
import de.bwaldvogel.liblinear.Parameter;
import de.bwaldvogel.liblinear.Problem;
import de.bwaldvogel.liblinear.SolverType;

/**
 * 
 * Title: 使用SVM的线性核进行训练 <br>
 * Description: <br>
 * Date: 2018年4月4日 <br>
 * Copyright (c) 2018 AILK <br>
 * 
 * @author liangyi
 */
public class AutoCatLibLinearTrainer {
    
    /**最多训练多少个样本*/
    static final int MAX_SAMPLE_NUMBER=Integer.MAX_VALUE;
    
    /**测试样本数所占总样本数的比例(大于等于0小于1)*/
    private double testSampleRatio=0.2;
    
    public AutoCatLibLinearTrainer() {
    }
    public AutoCatLibLinearTrainer(double testSampleRatio) {
        this.testSampleRatio=testSampleRatio;
    }
    
    /**
     * 使用样本进行训练
     * @return
     * @author liangyi
     */
    public Model train() throws Exception{
        long t1=System.currentTimeMillis();
        //加载样本到内存中
        SampleMapHolder<String, TrainningSampleVO> sampleMapHolder = loadTrainSampleMap();
        //将样本封装成svm所需的格式
        Problem trainningProblem=buildSvmProblem(sampleMapHolder.getTrainningSampleMap(),1);
        //封装测试样本
        Problem testProblem=buildSvmProblem(sampleMapHolder.getTestSampleMap(),1);
        long t2=System.currentTimeMillis();
        System.out.println("加载样本完毕，用时:"+(t2-t1)/1000+"秒");
        
        Model model=trainSample(trainningProblem,testProblem);
        return model;
    }
    
    public Model trainSample(Problem trainningProblem,Problem testProblem) throws Exception{
        Model bestModel=null;
        double bestRate=0;
        
        //使用不同的参数C和eps，获取最优解，返回最优的model
//        double[] cArray= {0.01,0.03,0.1,0.3,1,3,10};
//        double[] epsArray= {0.01,0.03,0.1,0.3,1,3,10};
        double[] cArray= {10};
        double[] epsArray= {0.01};
        List<VaildResultVO> resultList=new ArrayList<>(cArray.length*epsArray.length);
        
        //开始训练--------------------------
        System.out.println("开始训练样本");
        long t1=System.currentTimeMillis();
        for (int i = 0; i < cArray.length; i++) {
            for (int j = 0; j < epsArray.length; j++) {
                //训练的参数
                SolverType solver = SolverType.L2R_LR; // -s 0
                
                //Large C:Lower bias,high variance
                //Small C:Higher bias,low variance
                double C = cArray[i];    // cost of constraints violation
                double eps = epsArray[j]; // stopping criteria
                Parameter parameter = new Parameter(solver, C, eps);
                Model model = Linear.train(trainningProblem, parameter);
                
                //对训练样本进行测试，看看准确率
                VaildResultVO vaildResultVO=vaild(model,trainningProblem, testProblem, C, eps);
                
                resultList.add(vaildResultVO);
                
                if (vaildResultVO.trainningCorrectRate>bestRate) {
                    bestRate=vaildResultVO.trainningCorrectRate;
                    bestModel=model;
                }
                
                model=null;
            }
        }
        long t2=System.currentTimeMillis();
        System.out.println("训练样本完毕，用时:"+(t2-t1)/1000+"秒");
        
        //打印resultList
        Collections.sort(resultList);
        for (VaildResultVO vaildResultVO : resultList) {
            System.out.println(vaildResultVO);
        }
        
        return bestModel;
    }
    
    private VaildResultVO vaild(Model model,Problem trainningProblem,Problem testProblem,double C,double eps) throws Exception {
        //这里predictY要用实时训练出来的model
        double[] predictY=AutoCatPredictor.predictY(model,trainningProblem.x);
        int numberOfTrainningCorrect=0;
        for (int index = 0; index < predictY.length; index++) {
            if (predictY[index]==trainningProblem.y[index]) {
                numberOfTrainningCorrect++;
            }
        }
        double trainningCorrectRate=((double)numberOfTrainningCorrect)/predictY.length;
        
        //封装验证结果
        VaildResultVO vaildResultVO=new VaildResultVO();
        vaildResultVO.C=C;
        vaildResultVO.trainningCorrectRate=trainningCorrectRate;
        vaildResultVO.eps=eps;
        vaildResultVO.numberOfTrainningCorrect=numberOfTrainningCorrect;
        vaildResultVO.numberOfTrainningSample=predictY.length;
        
        if (testProblem!=null&&testProblem.l>0) {
            System.out.println("开始使用测试样本进行验证");
            predictY=AutoCatPredictor.predictY(model,testProblem.x);
            int numberOfTestCorrect=0;
            for (int index = 0; index < predictY.length; index++) {
                if (predictY[index]==trainningProblem.y[index]) {
                    numberOfTestCorrect++;
                }
            }
            double testCorrectRate=((double)numberOfTestCorrect)/predictY.length;
            
            vaildResultVO.testCorrectRate=testCorrectRate;
            vaildResultVO.numberOfTestCorrect=numberOfTestCorrect;
            vaildResultVO.numberOfTestSample=predictY.length;
        }
        
        return vaildResultVO;
    }

    /**
     * 加载样本数据
     * @return
     * @author liangyi
     * @throws Exception 
     */
    public SampleMapHolder<String, TrainningSampleVO> loadTrainSampleMap() throws Exception {
        System.out.println("开始加载样本");
        
        //读取样本文件获取所有样本信息-----------------------------------------------------------
        File orgSampleFile=FileUtil.getFileInSys(Constants.orgSampleFilePath);
        Map<String, TrainningSampleVO> sampleMap=new HashMap<>();
        try {
            BufferedReader e = new BufferedReader(new FileReader(orgSampleFile));
            for (String lineContent = e.readLine(); lineContent != null; lineContent = e.readLine()) {
                //解析每行数据
                int lastIndex=lineContent.lastIndexOf(",");
                if (lastIndex<=0) {
                    continue;
                }
                
                String gdsName=lineContent.substring(0, lastIndex);
                //商品分类
                String catId=lineContent.substring(lastIndex+1, lineContent.length());
                
                TrainningSampleVO sample=SampleFactory.buildTrainningSample(gdsName, catId,SampleInfoBuilderFactory.getBuilderForAutoCat());
                if (sample==null||sample.getY()<0) {
                    continue;
                }
                
                //要过滤掉相同的样本(样本相同的话，sample中yesIndexMap、y的内容必定是相同的)
                sampleMap.put(sample.getIndexCountAndYStr(), sample);
                
                if (sampleMap.size()>=MAX_SAMPLE_NUMBER) {
                    break;
                }
            }
            e.close();
        } catch (Exception e) {
            e.printStackTrace();
        }
        //读取样本文件获取所有样本信息end-----------------------------------------------------------
        System.out.println("从样本文件中获取到了"+sampleMap.size()+"个有效的样本");
        
        //测试样本所占比例为0.2
        SampleMapHolder<String, TrainningSampleVO> sampleMapHolder=new SampleMapHolder<>(testSampleRatio);
        for (Map.Entry<String, TrainningSampleVO> entry : sampleMap.entrySet()) {
            sampleMapHolder.addSample(entry.getKey(), entry.getValue());
        }
        System.out.println("训练样本数量为："+sampleMapHolder.getTrainningSampleMap().size());
        System.out.println("测试样本数量为："+sampleMapHolder.getTestSampleMap().size());
        
        return sampleMapHolder;
    }
    
    /**
     * 生成Problem
     * @param sampleMap
     * @param bias 线性回归的偏差值（一般是1，或0）
     * @return
     * @author liangyi
     */
    public Problem buildSvmProblem(Map<String, TrainningSampleVO> sampleMap,double bias) {
        //打印样本数量,key=y值，value=样本数
        Map<Integer, Integer> catSampleCountMap=new HashMap<>();
        
        //将样本封装成svm所需的格式
        Problem prob=new Problem();
        int xLength=DictMapOfAutoCat.getDictSize();
        /**训练数据的数目*/
        int l=sampleMap.size();
        /**包含其目标值的数组（分类时是整数，回归时是实数）*/
        double[] y=new double[l];
        /**一个指针的数组，每一个指针指向一个训练向量的离散表示（svm_node的数组）*/
        Feature[][] X=new FeatureNode[l][];
        
        //生成训练数据
        int i=0;
        for (Map.Entry<String, TrainningSampleVO> entry : sampleMap.entrySet()) {
            TrainningSampleVO sample=entry.getValue();
            
            Feature[] nodeArr=FeatureNodeBuilder.buildSvmNodes(sample);
            double yValue=sample.getY();
            
            X[i]=nodeArr;
            y[i]=yValue;
            
            i++;
            
            //为了打印样本数量
            Integer count=catSampleCountMap.get((int)yValue);
            if (count==null) {
                count=1;
            }else {
                count++;
            }
            catSampleCountMap.put((int)yValue, count);
        }
        prob.l=l;
        if (bias>0) {
            prob.n=xLength+1;
            prob.bias=bias;
        }else {
            prob.n=xLength;//bias=0，所以n=特征数
        }
        prob.x=X;
        prob.y=y;
        
        System.out.println("加载了"+prob.l+"个样本，共有"+catSampleCountMap.size()+"个分类：");
        for (Map.Entry<Integer, Integer> entry: catSampleCountMap.entrySet()) {
            int index=entry.getKey();
            int number=entry.getValue();
            CatInfo catInfo=CatMap.getCatInfoByIndex(index);
            if (catInfo==null) {
                continue;
            }
            System.out.println(catInfo.toString()+"，样本数量为 "+number);
        }
        
        return prob;
    }
    
}
